from DrissionPage import ChromiumPage
import requests
import time

# 目标网址和对应的类别
urls = [
    {"彩票": "https://www.news.cn/caipiao/index.html"},
    {"食品": "https://www.news.cn/food/index.html"},
    {"财经": "https://www.news.cn/fortune/index.htm"},
    {"汽车": "https://www.news.cn/auto/index.html"},
    {"书画": "https://www.news.cn/shuhua/index.html"},
    {"娱乐": "https://www.news.cn/ent/index.html"},
    {"科技": "https://www.news.cn/tech/index.html"},
    {"国际": "https://www.news.cn/world/index.html"},
    {"健康": "https://www.news.cn/health/index.html"},
    {"旅游": "https://www.news.cn/travel/index.html"},
    {"教育": "https://education.news.cn/jcjy/index.htm"},
    {"能源": "https://www.news.cn/energy/index.html"},
    {"时尚": "https://www.news.cn/fashion/index.html"},
    {"港澳": "https://www.news.cn/gangao/index.html"},
    {"台湾": "https://www.news.cn/tw/index.html"}
]


# 类别到数字的映射
category_to_number = {
    "彩票": 0,
    "食品": 1,
    "财经": 2,
    "汽车": 3,
    "书画": 4,
    "娱乐": 5,
    "科技": 6,
    "国际": 7,
    "健康": 8,
    "旅游": 9,
    "教育": 10,
    "能源": 11,
    "时尚": 12,
    "港澳": 13,
    "台湾": 14
}


# 初始化浏览器页面
page = ChromiumPage().latest_tab

# 打开文件准备写入
with open('news_titles.txt', 'w', encoding='utf-8') as file:
    for url_dict in urls:
        # 获取类别名称和对应的 URL
        category = list(url_dict.keys())[0]
        url = list(url_dict.values())[0]

        # 打印日志信息，告知当前正在处理哪个类别
        print(f"正在抓取类别: {category}（对应数字: {category_to_number.get(category)}）")

        # 启动监听，匹配以 ds 开头并以 .json 结尾的 URL
        page.listen.start('[a-z/.:]+ds_[0-9a-z]+.json', is_regex=True)

        # 获取页面
        page.get(url)

        # 查找并点击更多按钮
        button = page.ele("@text()=加载更多")
        time.sleep(0.5)
        button.click()

        # 获取请求的 URL
        data_url = page.listen.wait().url

        # 请求并解析数据
        response = requests.get(data_url)
        data = response.json().get('datasource', [])

        # 获取每个标题并写入文件
        for item in data:
            title = item.get('title', '')  # 获取标题，防止缺失
            # 将类别名称转换为数字
            category_number = category_to_number.get(category, 0)  # 默认为 0
            # 写入文件，每一行：类别数字, 标题
            file.write(f"{title.strip()} {category_number}\n")
        print(f"类别: {category}抓取完成，共{len(data)}条数据")
print("数据写入完成！")
